package com.chance.cc.crawler.development.bootstrap.xiaohongshu;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;

import java.io.*;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_Keword;
import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

public class XHSCustomStart {

    private static final String domainId = "xhs";
    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("H7307T4706B25G4D");
        proxy.setPassword("05B4877CC39192C0");
    }


    public static void main(String[] args) throws IOException, ParseException, NoSuchAlgorithmException {

//        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
//                .startPageRequest(domainId, turnPage)
//                .httpUrl("/aweme/v1/general/search/single")
//                .releaseTime(System.currentTimeMillis())
//                .filter(CrawlerEnum.CrawlerRecordFilter.key)
//                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
//                .needWashed(false)
//                .needParsed(false)
//                .build();
//        requestRecord.setDownload(false);
//        requestRecord.setSkipPipeline(true);


        xhsNoteInfoImport();
//        xhsNoteInfoUniqe();
//        test1();
//        testEmail();
//        testSchedulerEmail();
//        ungzip();
    }

    private static void test(){
        RedisReader redisReader = new RedisReader("192.168.1.215",6379,3);
        CrawlerRequestRecord crawlerRequestRecord = redisReader.recordPop("crawler-xhs-queue_list", "crawler-xhs-queue_map");



        CrawlerRequestRecord userOauthInfoCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xhs_user_oauth_infos", turnPageItem)
                .httpUrl("http://192.168.1.215:9599/crawler/oauth/api/v1/xhs/userOauthInfos")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline() //控制台输t出
                .fileResultPipeline("D:\\chance\\log\\dy.json",false)
                .requestRecord(crawlerRequestRecord)
                .supportRecord(userOauthInfoCrawlerRecord)
                .build()
                .start();
    }

    private static void test1(){
        RedisReader redisReader = new RedisReader("192.168.1.215",6379,3);
        CrawlerRequestRecord crawlerRequestRecord = redisReader.recordPop("crawler-xhs-custom-queue_list", "crawler-xhs-custom-queue_map");

        crawlerRequestRecord.tagsCreator().bizTags().addSite("search_custom");

        CrawlerRequestRecord userOauthInfoCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xhs_user_oauth_infos", turnPageItem)
                .httpUrl("http://192.168.1.215:9599/crawler/oauth/api/v1/xhs/userOauthInfos")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(domainId,"cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline() //控制台输t出
                .fileResultPipeline("D:\\chance\\log\\dy.json",false)
                .requestRecord(crawlerRequestRecord)
                .supportRecord(userOauthInfoCrawlerRecord)
                .build()
                .start();
    }

    private static void testEmail(){


        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xhs-monitor", turnPage)
                .httpUrl("/node/xhs-monitor")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(true)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addCustomKV("error_count",7);

        DevCrawlerController.builder()
                .triggerInfo("xhs-monitor","cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline() //控制台输t出
                .requestRecord(requestRecord)
                .build()
                .start();
    }

    private static void testSchedulerEmail() throws FileNotFoundException {
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xhs-monitor", turnPage)
                .httpUrl("/scheduler/xhs-monitor")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(true)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);


        DevCrawlerController.builder()
                .triggerInfo("xhs-monitor","cron",System.currentTimeMillis(),"realtime")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(StringUtils.joinWith("-","crawler",domainId,"queue"))) //内存队列
                .consoleResultPipeline() //控制台输t出
                .requestRecord(requestRecord)
                .build()
                .start();
    }

    private static void ungzip() throws IOException {
//        GZIPInputStream gzipInputStream = new GZIPInputStream(new FileInputStream("C:\\Users\\Administrator\\Desktop\\jar\\test"));
//
//        List<String> strings = IOUtils.readLines(gzipInputStream, "utf-8");
//
//        System.out.println(JSON.toJSONString(strings));
        List<String> bytestr = IOUtils.readLines(new FileInputStream("C:\\Users\\Administrator\\Desktop\\jar\\test"), "UTF-8");
        String[] split = bytestr.get(0).split(",");
        byte[] bytes = new byte[split.length];
        for (int i = 0; i < split.length; i++) {
            bytes[i] = (byte)Integer.parseInt(split[i]);
        }
        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
        GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream);
        List<String> strings = IOUtils.readLines(gzipInputStream, "utf-8");
//
        System.out.println(JSON.toJSONString(strings));

    }

    private static void xhsNoteInfoImport() throws IOException, NoSuchAlgorithmException, ParseException {
        RedisReader redisReader = new RedisReader("192.168.1.214",6379,3);

        List<String> readLines = IOUtils.readLines(new FileInputStream("D:\\chance\\data\\xhs\\美妆-按文章采集-20210629 (1).txt"), "utf-8");
        for (String readLine : readLines) {
            try {
                if (readLine.startsWith("KOL")){
                    continue;
                }
                String[] split = readLine.split("\t");
                String releaseTime = split[4];
                String noteUrl = split[6];
                String[] urls = noteUrl.split("/");
                String noteId = urls[urls.length -1];
                String keyword = split[8];
                keyword = keyword.replaceAll("\"","");

//            String searchUrl = "https://www.xiaohongshu.com/api/sns/v10/search/notes?keyword="+keyword;
                String wxappArticleUrlFormat = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/note/%s/single_feed";

                String articleUrl = String.format(wxappArticleUrlFormat, noteId);
                String urlPath = articleUrl.substring(articleUrl.indexOf("/fe_api/"));
                String x_sign = "X" + md5(urlPath + "WSUDD");

                CrawlerRequestRecord crawlerArticleRequestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest("xhs",turnPageItem)
                        .httpUrl(articleUrl)
                        .httpHead("x-sign",x_sign)
                        .releaseTime(dateToTimestamp(releaseTime))
                        .needWashed(true)
                        .proxy(proxy)
                        .build();
                crawlerArticleRequestRecord.tagsCreator().bizTags().addCustomKV(Tag_Field_Keword,keyword);


                crawlerArticleRequestRecord.tagsCreator().bizTags().addCustomKV("noteId",noteId);
                crawlerArticleRequestRecord.tagsCreator().bizTags().addCustomKV("releaseTime",releaseTime);
                crawlerArticleRequestRecord.tagsCreator().bizTags().addSite("search_custom");
                crawlerArticleRequestRecord.tagsCreator().bizTags().addSiteBiz("search_custom");
                crawlerArticleRequestRecord.tagsCreator().bizTags().addDomain("xhs");

                String requestKey = crawlerArticleRequestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);
                //push kv first

                redisReader.mapPush("crawler-xhs-custom-queue_map",shaKey, JSON.toJSONString(crawlerArticleRequestRecord));
                redisReader.listPush("crawler-xhs-custom-queue_list",shaKey);

            } catch (Exception e) {
                System.out.println();
                e.printStackTrace();
            }
        }
    }

    private static void xhsNoteInfoUniqe() throws IOException {
        List<String> readLines = IOUtils.readLines(new FileInputStream("C:\\Users\\Administrator\\Desktop\\jar\\xhs\\xhs_topic.txt"), "GBK");
        List<String> noteIds = new ArrayList<>();
        List<String> results = new ArrayList<>();
        for (String readLine : readLines) {
            if (readLine.startsWith("KOL")){
                continue;
            }
            String[] split = readLine.split("\t");
            String noteUrl = split[6];
            String[] urls = noteUrl.split("/");
            String noteId = urls[urls.length -1];

            if (noteIds.contains(noteId)){
//                noteIds.add(noteId);
                results.add(readLine);
            } else {
                noteIds.add(noteId);
            }

        }

        IOUtils.writeLines(results,"\n",new FileOutputStream("C:\\Users\\Administrator\\Desktop\\jar\\xhs\\all_data_1_new.txt"),"utf-8");
    }


    public static long dateToTimestamp(String dataStr) throws ParseException {
        String regEx="[^0-9]+"; //正则表达式，用于匹配非数字串，+号用于匹配出多个非数字串
        Pattern pattern = Pattern.compile(regEx);

        if (dataStr.equals("刚刚")){
            return System.currentTimeMillis();
        } else if (Pattern.matches("\\d*秒前",dataStr)){
            int number = Integer.valueOf(pattern.split(dataStr)[0]);
            return (System.currentTimeMillis() - number*1000L);
        } else if (Pattern.matches("\\d*分钟前",dataStr)){
            int number = Integer.valueOf(pattern.split(dataStr)[0]);
            return (System.currentTimeMillis() - number*60*1000L);
        } else if (Pattern.matches("\\d*小时前",dataStr)){
            int number = Integer.valueOf(pattern.split(dataStr)[0]);
            return (System.currentTimeMillis() - number*60*60*1000L);
        } else if (Pattern.matches("\\d*小时\\d*分钟前",dataStr)){
            int hourNumber = Integer.valueOf(pattern.split(dataStr)[0]);
            int secondNumber = Integer.valueOf(pattern.split(dataStr)[1]);
            long totalMillis = hourNumber*60*60*1000L + secondNumber*60*1000L;
            return (System.currentTimeMillis() - totalMillis);
        } else if (dataStr.startsWith("今天")){
            String currentTime = DateFormatUtils.format(System.currentTimeMillis(), TimeForamtEnum.format1.getFormat());
            dataStr = dataStr.replace("今天", currentTime);
        } else if (dataStr.startsWith("昨天")){
            String yesterdayTime = DateFormatUtils.format(System.currentTimeMillis() - 60*60*24*1000L, TimeForamtEnum.format1.getFormat());
            dataStr = dataStr.replace("昨天", yesterdayTime);
        } else if (dataStr.startsWith("前天")){
            String beforeYesterdayTime = DateFormatUtils.format(System.currentTimeMillis() - 2*60*60*24*1000L, TimeForamtEnum.format1.getFormat());
            dataStr = dataStr.replace("前天", beforeYesterdayTime);
        } else if (Pattern.matches("\\d{2}月\\d{2}[日]*", dataStr)){
            Calendar cal = Calendar.getInstance();
            dataStr = cal.get(Calendar.YEAR)+"年"+dataStr;
        } else if (Pattern.matches("\\d{2}-\\d{2}", dataStr)){
            Calendar cal = Calendar.getInstance();
            dataStr = cal.get(Calendar.YEAR)+"-"+dataStr;
        }
        return DateUtils.parseDateStrictly(dataStr, TimeForamtEnum.allFormats()).getTime();
    }

    public enum TimeForamtEnum {
        format1("yyyy年MM月dd日"),
        format1_1("yyyy年MM月dd日 HH:mm:ss"),
        format1_2("yyyy年MM月dd日 HH:mm"),
        format1_3("yyyy年MM月dd日HH:mm"),
        format1_4("yyyy年MM月dd日 HH点mm分"),
        format1_5("yyyy年MM月dd日 HH点mm分ss秒"),
        format1_6("yyyy年MM月dd日HH点mm分"),
        format1_7("yyyy年MM月dd日HH点mm分ss秒"),


        format4("yyyy-MM-dd HH:mm:ss"),
        format5("yyyy-MM-dd HH:mm"),
        format6("yyyy-MM-dd"),

        format7("yyyy/MM/dd HH:mm:ss"),
        format7_1("MM/dd/yyyy HH:mm:ss"),
        format8("yyyy/MM/dd HH:mm"),
        format9("yyyy/MM/dd"),

        format10("yyyy.MM.dd HH:mm:ss"),
        format11("yyyy.MM.dd HH:mm"),
        format12("yyyy.MM.dd"),

        format13("EEE MMM d HH:mm:ss +0800 yyyy");

        private String format;

        private TimeForamtEnum(String format) {
            this.format = format;
        }

        public static String[] allFormats() {
            TimeForamtEnum[] timeForamtEnums = TimeForamtEnum.values();
            String[] formats = new String[timeForamtEnums.length];
            for (int count = 0; count < timeForamtEnums.length; count++) {
                formats[count] = timeForamtEnums[count].format;
            }
            return formats;
        }

        public String getFormat() {
            return format;
        }
    }

}
